import requests
from bs4 import BeautifulSoup
if __name__=='__main__':
    url='https://www.shicimingju.com/book/sanguoyanyi.html'
    header={'user-agent':'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/92.0.4515.159 Safari/537.36'}
    page_obj=requests.get(url=url,headers=header)
    page_obj.encoding='utf-8'
    page_text=page_obj.text
    soup=BeautifulSoup(page_text,'lxml')
    li_list = soup.select('.book-mulu > ul > li')
    filename= '三国演义.txt'
    with open(filename,'w',encoding='utf-8') as fp:
        for li in li_list:
            detail_title=li.text
            detail_url='https://www.shicimingju.com'+li.a['href']
            detail_obj=requests.get(url=detail_url,headers=header)
            detail_obj.encoding='utf-8'
            print(detail_url)
            detail_page=detail_obj.text
            detail_soup=BeautifulSoup(detail_page,'lxml')
            detail_text=detail_soup.find('div',class_='chapter_content').text
            fp.write(detail_title+'\n')
            fp.write(detail_text+'\n')

